{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "RealTimePanoptic-TensorRT-version.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyMQKdAF+mu3xeMe59EsQwdH",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/pushkar-khetrapal/RealTimePanoptic-TensorRT/blob/master/RealTimePanoptic_TensorRT_version.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oPRlgnDFizCO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!nvidia-smi"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hfTfckPFi9j9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Uninstall the current CUDA version\n",
        "!sudo apt-get --purge remove cuda nvidia* libnvidia-*\n",
        "!sudo dpkg -l | grep cuda- | awk '{print $2}' | xargs -n1 dpkg --purge\n",
        "!sudo apt-get remove cuda-*\n",
        "!sudo apt autoremove"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kBO1Br4-jD2j",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Installing CUDA 10.0\n",
        "\n",
        "!wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/cuda-repo-ubuntu1804_10.0.130-1_amd64.deb\n",
        "!sudo dpkg -i cuda-repo-ubuntu1804_10.0.130-1_amd64.deb\n",
        "!sudo apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub\n",
        "!sudo apt-get update\n",
        "!wget http://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64/nvidia-machine-learning-repo-ubuntu1804_1.0.0-1_amd64.deb\n",
        "!sudo apt install -y ./nvidia-machine-learning-repo-ubuntu1804_1.0.0-1_amd64.deb\n",
        "!sudo apt-get update\n",
        "\n",
        "# Install NVIDIA driver\n",
        "!sudo apt-get -y installnvidia-driver-418\n",
        "\n",
        "# Install development and runtime libraries (~4GB)\n",
        "!sudo apt-get install -y \\\n",
        "    cuda-10-0 \\\n",
        "    libcudnn7=7.6.2.24-1+cuda10.0  \\\n",
        "    libcudnn7-dev=7.6.2.24-1+cuda10.0 --allow-change-held-packages"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GNscoBEijIo_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!nvcc --version"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5kp_OZecjLxl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!wget 'https://www.dropbox.com/s/1qadpocgzspmb6j/nv-tensorrt-repo-ubuntu1804-cuda10.0-trt7.0.0.11-ga-20191216_1-1_amd64.deb?dl=0'\n",
        "!mv '/content/nv-tensorrt-repo-ubuntu1804-cuda10.0-trt7.0.0.11-ga-20191216_1-1_amd64.deb?dl=0' '/content/nv-tensorrt-repo-ubuntu1804-cuda10.0-trt7.0.0.11-ga-20191216_1-1_amd64.deb'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fe-33tMfkCUf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!sudo dpkg -i \"/content/nv-tensorrt-repo-ubuntu1804-cuda10.0-trt7.0.0.11-ga-20191216_1-1_amd64.deb\"\n",
        "!sudo apt-key add /var/nv-tensorrt-repo-cuda10.0-trt7.0.0.11-ga-20191216/7fa2af80.pub\n",
        "!sudo apt-get update"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X4unDCZdkGYt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!sudo apt-get install libnvinfer7=7.0.0-1+cuda10.0 libnvonnxparsers7=7.0.0-1+cuda10.0 libnvparsers7=7.0.0-1+cuda10.0 libnvinfer-plugin7=7.0.0-1+cuda10.0 libnvinfer-dev=7.0.0-1+cuda10.0 libnvonnxparsers-dev=7.0.0-1+cuda10.0 libnvparsers-dev=7.0.0-1+cuda10.0 libnvinfer-plugin-dev=7.0.0-1+cuda10.0 python-libnvinfer=7.0.0-1+cuda10.0 python3-libnvinfer=7.0.0-1+cuda10.0\n",
        "!sudo apt-mark hold libnvinfer7 libnvonnxparsers7 libnvparsers7 libnvinfer-plugin7 libnvinfer-dev libnvonnxparsers-dev libnvparsers-dev libnvinfer-plugin-dev python-libnvinfer python3-libnvinfer\n",
        "!pip install pycuda \n",
        "!pip install onnx\n",
        "!sudo apt-get install tensorrt"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WWrURDY8kNLS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!dpkg -l | grep TensorRT"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F4g-hcOCznWF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "## installing dependencies\n",
        "%%shell\n",
        "git clone https://github.com/pushkar-khetrapal/RealTimePanoptic-TensorRT.git\n",
        "wget https://wallup.net/wp-content/uploads/2016/01/256862-car-landscape-vehicle-road.jpg\n",
        "wget https://tri-ml-public.s3.amazonaws.com/github/realtime_panoptic/models/cvpr_realtime_pano_cityscapes_standalone_no_prefix.pth\n",
        "pip install yacs\n",
        "pip uninstall -y apex\n",
        "git clone https://www.github.com/nvidia/apex\n",
        "cd apex\n",
        "python setup.py install\n",
        "mv '/content/RealTimePanoptic-TensorRT/realtime_panoptic' '/usr/local/lib/python3.6/dist-packages'\n",
        "## attach the link of image here\n",
        "wget 'https://storage.googleapis.com/groundai-web-prod/media%2Fusers%2Fuser_272722%2Fproject_255779%2Fimages%2Ftraining_0.jpg' "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HgGzlmI7w_Tz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "''' The model contains :\n",
        "\n",
        "\"Making onnx model for\"(\n",
        "Backone\n",
        "panoptic head\n",
        ")\n",
        "and returns (locations, box_cls, box_regression, centerness, levelness_logits, semantic_logits, interpolated_semantic_logits)\n",
        "\n",
        "\n",
        "Further these (locations, box_cls, box_regression, centerness, levelness_logits, semantic_logits, interpolated_semantic_logits)\n",
        "feed into PanopticFromDenseBox.process.\n",
        "\n",
        "'''\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IK_ZJflSzwaR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "## importing libraries\n",
        "import warnings\n",
        "import argparse\n",
        "import torch\n",
        "import numpy as np\n",
        "from PIL import Image\n",
        "from torchvision.models.detection.image_list import ImageList\n",
        "import time\n",
        "\n",
        "from realtime_panoptic.models.rt_pano_net import RTPanoNet\n",
        "from realtime_panoptic.config import cfg\n",
        "import realtime_panoptic.data.panoptic_transform as P\n",
        "from realtime_panoptic.utils.visualization import visualize_segmentation_image, visualize_detection_image\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "cityscapes_colormap = np.array([\n",
        " [128,  64, 128],\n",
        " [244,  35, 232],\n",
        " [ 70,  70,  70],\n",
        " [102, 102, 156],\n",
        " [190, 153, 153],\n",
        " [153, 153, 153],\n",
        " [250 ,170,  30],\n",
        " [220, 220,   0],\n",
        " [107, 142,  35],\n",
        " [152, 251, 152],\n",
        " [ 70, 130, 180],\n",
        " [220,  20,  60],\n",
        " [255,   0,   0],\n",
        " [  0,   0, 142],\n",
        " [  0,   0,  70],\n",
        " [  0,  60, 100],\n",
        " [  0,  80, 100],\n",
        " [  0,   0, 230],\n",
        " [119,  11,  32],\n",
        " [  0,   0,   0]])\n",
        "\n",
        "cityscapes_instance_label_name = ['person', 'rider', 'car', 'truck', 'bus', 'train', 'motorcycle', 'bicycle']\n",
        "warnings.filterwarnings(\"ignore\", category=UserWarning)\n",
        "cfg.merge_from_file('RealTimePanoptic-TensorRT/configs/demo_config.yaml')\n",
        "\n",
        "## developing model\n",
        "model = RTPanoNet(\n",
        "    backbone=cfg.model.backbone, \n",
        "    num_classes=cfg.model.panoptic.num_classes,\n",
        "    things_num_classes=cfg.model.panoptic.num_thing_classes,\n",
        "    pre_nms_thresh=cfg.model.panoptic.pre_nms_thresh,\n",
        "    pre_nms_top_n=cfg.model.panoptic.pre_nms_top_n,\n",
        "    nms_thresh=cfg.model.panoptic.nms_thresh,\n",
        "    fpn_post_nms_top_n=cfg.model.panoptic.fpn_post_nms_top_n,\n",
        "    instance_id_range=cfg.model.panoptic.instance_id_range)\n",
        "device = 'cuda'\n",
        "model.to(device)\n",
        "\n",
        "model.load_state_dict(torch.load('cvpr_realtime_pano_cityscapes_standalone_no_prefix.pth'))\n",
        "\n",
        "output_names = [\"output_0\"] + [\"output_%d\" % i for i in range(1,24)]\n",
        "input_names = [\"input1\"]\n",
        "\n",
        "x = torch.randn(1, 3, 1024, 2048, requires_grad=True).to('cuda')\n",
        "torch.onnx.export(model, x, \"model.onnx\", verbose=True, input_names=input_names, output_names=output_names)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CRILzbGtmVsM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import tensorrt as trt \n",
        "import torch.onnx as onnx\n",
        "import numpy as np \n",
        "import pycuda.autoinit\n",
        "import time\n",
        "import cv2 as cv\n",
        "#import onnx\n",
        "\n",
        "import pycuda.driver as cuda\n",
        "\n",
        "model_path = \"/content/model.onnx\"\n",
        "# onnx.checker.check_model(model_path)\n",
        "# onnx.helper.printable_graph(model.graph)\n",
        "\n",
        "input_size = 224\n",
        "\n",
        "TRT_LOGGER = trt.Logger(trt.Logger.WARNING)       \n",
        "network_flags = 1 << int(trt.NetworkDefinitionCreationFlag.EXPLICIT_BATCH)\n",
        "def build_engine(model_path):\n",
        "    with trt.Builder(TRT_LOGGER) as builder, builder.create_network(flags = network_flags) as network, \\\n",
        "    trt.OnnxParser(network, TRT_LOGGER) as parser:\n",
        "        builder.max_workspace_size = 1<<int(trt.NetworkDefinitionCreationFlag.EXPLICIT_BATCH)\n",
        "        builder.max_batch_size = 1\n",
        "        builder.fp16_mode = 1\n",
        "\n",
        "        \n",
        "        with open(model_path, 'rb') as f:\n",
        "            value = parser.parse(f.read())\n",
        "            print(\"Parser: \", value)\n",
        "            if (value == False):\n",
        "                print(trt.ParserError().code())\n",
        "            \n",
        "        engine = builder.build_cuda_engine(network)\n",
        "        print(engine)\n",
        "        return engine\n",
        "\n",
        "def alloc_buf(engine):\n",
        "    h_in_size = trt.volume(engine.get_binding_shape(0))\n",
        "    h_out_size = trt.volume(engine.get_binding_shape(1))\n",
        "    h_in_dtype = trt.nptype(engine.get_binding_dtype(0))\n",
        "    h_out_dtype = trt.nptype(engine.get_binding_dtype(1))\n",
        "    h_in_dtype = trt.nptype(engine.get_binding_dtype(0))\n",
        "    h_out_dtype = trt.nptype(engine.get_binding_dtype(1))\n",
        "    in_cpu = cuda.pagelocked_empty(h_in_size, h_in_dtype)\n",
        "    out_cpu = cuda.pagelocked_empty(h_out_size, h_out_dtype)\n",
        "    in_gpu = cuda.mem_alloc(in_cpu.nbytes)\n",
        "    out_gpu = cuda.mem_alloc(out_cpu.nbytes)\n",
        "    stream = cuda.Stream()\n",
        "    return in_cpu, out_cpu, in_gpu, out_gpu, stream\n",
        "\n",
        "def inference(engine, context, inputs, out_cpu, in_gpu, out_gpu, stream):\n",
        "    cuda.memcpy_htod(in_gpu, inputs)\n",
        "    context.execute(bindings=[int(in_gpu), int(out_gpu)])\n",
        "    cuda.memcpy_dtoh(out_cpu, out_gpu)\n",
        "    return out_cpu\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "    inputs = np.random.random((1, 3, 1024, 2048)).astype(np.float32)\n",
        "    engine = build_engine(model_path)\n",
        "    print(engine)\n",
        "    context = engine.create_execution_context()\n",
        "    with open(\"AlexNet.engine\", \"wb\") as f:\n",
        "        f.write(engine.serialize())\n",
        "    for _ in range(10):\n",
        "        t1 = time.time()\n",
        "        in_cpu, out_cpu, in_gpu, out_gpu, stream = alloc_buf(engine)\n",
        "        print(inputs.shape)\n",
        "        res = inference(engine, context, inputs.reshape(-1), out_cpu, in_gpu, out_gpu, stream)\n",
        "    print(len(res))\n",
        "    print(\"cost time: \", time.time()-t1)\n"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}